{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 4.数据变换\n",
    "## 4.1 对于标称型属性"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0    208500\n",
      "1    181500\n",
      "2    223500\n",
      "3    140000\n",
      "4    250000\n",
      "5    143000\n",
      "6    307000\n",
      "7    200000\n",
      "8    129900\n",
      "9    118000\n",
      "Name: SalePrice, dtype: int64\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>MSSubClass</th>\n",
       "      <th>MSZoning</th>\n",
       "      <th>LotFrontage</th>\n",
       "      <th>LotArea</th>\n",
       "      <th>Street</th>\n",
       "      <th>Alley</th>\n",
       "      <th>LotShape</th>\n",
       "      <th>LandContour</th>\n",
       "      <th>Utilities</th>\n",
       "      <th>LotConfig</th>\n",
       "      <th>...</th>\n",
       "      <th>ScreenPorch</th>\n",
       "      <th>PoolArea</th>\n",
       "      <th>PoolQC</th>\n",
       "      <th>Fence</th>\n",
       "      <th>MiscFeature</th>\n",
       "      <th>MiscVal</th>\n",
       "      <th>MoSold</th>\n",
       "      <th>YrSold</th>\n",
       "      <th>SaleType</th>\n",
       "      <th>SaleCondition</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>60</td>\n",
       "      <td>RL</td>\n",
       "      <td>65.0</td>\n",
       "      <td>8450</td>\n",
       "      <td>Pave</td>\n",
       "      <td>None</td>\n",
       "      <td>Reg</td>\n",
       "      <td>Lvl</td>\n",
       "      <td>AllPub</td>\n",
       "      <td>Inside</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>2008</td>\n",
       "      <td>WD</td>\n",
       "      <td>Normal</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>20</td>\n",
       "      <td>RL</td>\n",
       "      <td>80.0</td>\n",
       "      <td>9600</td>\n",
       "      <td>Pave</td>\n",
       "      <td>None</td>\n",
       "      <td>Reg</td>\n",
       "      <td>Lvl</td>\n",
       "      <td>AllPub</td>\n",
       "      <td>FR2</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>0</td>\n",
       "      <td>5</td>\n",
       "      <td>2007</td>\n",
       "      <td>WD</td>\n",
       "      <td>Normal</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>60</td>\n",
       "      <td>RL</td>\n",
       "      <td>68.0</td>\n",
       "      <td>11250</td>\n",
       "      <td>Pave</td>\n",
       "      <td>None</td>\n",
       "      <td>IR1</td>\n",
       "      <td>Lvl</td>\n",
       "      <td>AllPub</td>\n",
       "      <td>Inside</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>0</td>\n",
       "      <td>9</td>\n",
       "      <td>2008</td>\n",
       "      <td>WD</td>\n",
       "      <td>Normal</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>70</td>\n",
       "      <td>RL</td>\n",
       "      <td>60.0</td>\n",
       "      <td>9550</td>\n",
       "      <td>Pave</td>\n",
       "      <td>None</td>\n",
       "      <td>IR1</td>\n",
       "      <td>Lvl</td>\n",
       "      <td>AllPub</td>\n",
       "      <td>Corner</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>2006</td>\n",
       "      <td>WD</td>\n",
       "      <td>Abnorml</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>60</td>\n",
       "      <td>RL</td>\n",
       "      <td>84.0</td>\n",
       "      <td>14260</td>\n",
       "      <td>Pave</td>\n",
       "      <td>None</td>\n",
       "      <td>IR1</td>\n",
       "      <td>Lvl</td>\n",
       "      <td>AllPub</td>\n",
       "      <td>FR2</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>0</td>\n",
       "      <td>12</td>\n",
       "      <td>2008</td>\n",
       "      <td>WD</td>\n",
       "      <td>Normal</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 79 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   MSSubClass MSZoning  LotFrontage  LotArea Street Alley LotShape  \\\n",
       "0          60       RL         65.0     8450   Pave  None      Reg   \n",
       "1          20       RL         80.0     9600   Pave  None      Reg   \n",
       "2          60       RL         68.0    11250   Pave  None      IR1   \n",
       "3          70       RL         60.0     9550   Pave  None      IR1   \n",
       "4          60       RL         84.0    14260   Pave  None      IR1   \n",
       "\n",
       "  LandContour Utilities LotConfig  ... ScreenPorch PoolArea PoolQC Fence  \\\n",
       "0         Lvl    AllPub    Inside  ...           0        0   None  None   \n",
       "1         Lvl    AllPub       FR2  ...           0        0   None  None   \n",
       "2         Lvl    AllPub    Inside  ...           0        0   None  None   \n",
       "3         Lvl    AllPub    Corner  ...           0        0   None  None   \n",
       "4         Lvl    AllPub       FR2  ...           0        0   None  None   \n",
       "\n",
       "  MiscFeature MiscVal  MoSold  YrSold  SaleType  SaleCondition  \n",
       "0        None       0       2    2008        WD         Normal  \n",
       "1        None       0       5    2007        WD         Normal  \n",
       "2        None       0       9    2008        WD         Normal  \n",
       "3        None       0       2    2006        WD        Abnorml  \n",
       "4        None       0      12    2008        WD         Normal  \n",
       "\n",
       "[5 rows x 79 columns]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train = pd.read_csv(\"./clean_data/train_afterclean.csv\")\n",
    "print(train.SalePrice.head(10))\n",
    "test = pd.read_csv(\"./clean_data/test_afterclean.csv\")\n",
    "alldata = pd.concat((train.loc[:,'MSSubClass':'SaleCondition'], test.loc[:,'MSSubClass':'SaleCondition']), ignore_index=True)\n",
    "alldata.shape\n",
    "alldata.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 处理序列型标称数据\n",
    "ordinalList = ['ExterQual', 'ExterCond', 'GarageQual', 'GarageCond','PoolQC',\\\n",
    "              'FireplaceQu', 'KitchenQual', 'HeatingQC', 'BsmtQual','BsmtCond']\n",
    "ordinalmap = {'Ex': 5,'Gd': 4,'TA': 3,'Fa': 2,'Po': 1,'None': 0}\n",
    "for c in ordinalList:\n",
    "    alldata[c] = alldata[c].map(ordinalmap) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 其他类别属性——标签化\n",
    "alldata['BsmtExposure'] = alldata['BsmtExposure'].map({'None':0, 'No':1, 'Mn':2, 'Av':3, 'Gd':4})    \n",
    "alldata['BsmtFinType1'] = alldata['BsmtFinType1'].map({'None':0, 'Unf':1, 'LwQ':2,'Rec':3, 'BLQ':4, 'ALQ':5, 'GLQ':6})\n",
    "alldata['BsmtFinType2'] = alldata['BsmtFinType2'].map({'None':0, 'Unf':1, 'LwQ':2,'Rec':3, 'BLQ':4, 'ALQ':5, 'GLQ':6})\n",
    "alldata['Functional'] = alldata['Functional'].map({'Maj2':1, 'Sev':2, 'Min2':3, 'Min1':4, 'Maj1':5, 'Mod':6, 'Typ':7})\n",
    "alldata['GarageFinish'] = alldata['GarageFinish'].map({'None':0, 'Unf':1, 'RFn':2, 'Fin':3})\n",
    "alldata['Fence'] = alldata['Fence'].map({'MnWw':0, 'GdWo':1, 'MnPrv':2, 'GdPrv':3, 'None':4})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 部分属性创建二值新属性\n",
    "MasVnrType_Any = alldata.MasVnrType.replace({'BrkCmn': 1,'BrkFace': 1,'CBlock': 1,'Stone': 1,'None': 0})\n",
    "MasVnrType_Any.name = 'MasVnrType_Any' #修改该series的列名\n",
    "SaleCondition_PriceDown = alldata.SaleCondition.replace({'Abnorml': 1,'Alloca': 1,'AdjLand': 1,'Family': 1,'Normal': 0,'Partial': 0})\n",
    "SaleCondition_PriceDown.name = 'SaleCondition_PriceDown' #修改该series的列名"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "alldata = alldata.replace({'CentralAir': {'Y': 1,'N': 0}})\n",
    "alldata = alldata.replace({'PavedDrive': {'Y': 1,'P': 0,'N': 0}})\n",
    "newer_dwelling = alldata['MSSubClass'].map({20: 1,30: 0,40: 0,45: 0,50: 0,60: 1,70: 0,75: 0,80: 0,85: 0,90: 0,120: 1,150: 0,160: 0,180: 0,190: 0})\n",
    "newer_dwelling.name= 'newer_dwelling' #修改该series的列名\n",
    "alldata['MSSubClass'] = alldata['MSSubClass'].apply(str)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "Neighborhood_Good = pd.DataFrame(np.zeros((alldata.shape[0],1)), columns=['Neighborhood_Good'])\n",
    "Neighborhood_Good[alldata.Neighborhood=='NridgHt'] = 1\n",
    "Neighborhood_Good[alldata.Neighborhood=='Crawfor'] = 1\n",
    "Neighborhood_Good[alldata.Neighborhood=='StoneBr'] = 1\n",
    "Neighborhood_Good[alldata.Neighborhood=='Somerst'] = 1\n",
    "Neighborhood_Good[alldata.Neighborhood=='NoRidge'] = 1\n",
    "# Neighborhood_Good = (alldata['Neighborhood'].isin(['StoneBr','NoRidge','NridgHt','Timber','Somerst']))*1 #(效果没有上面好)\n",
    "Neighborhood_Good.name='Neighborhood_Good'# 将该变量加入"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "season = (alldata['MoSold'].isin([5,6,7]))*1 #(@@@@@)\n",
    "season.name='season'\n",
    "alldata['MoSold'] = alldata['MoSold'].apply(str)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 对与“质量——Qual”“条件——Cond”属性，构造新属性\n",
    "# 处理OverallQual：将该属性分成两个子属性，以5为分界线，大于5及小于5的再分别以序列\n",
    "overall_poor_qu = alldata.OverallQual.copy()# Series类型\n",
    "overall_poor_qu = 5 - overall_poor_qu\n",
    "overall_poor_qu[overall_poor_qu<0] = 0\n",
    "overall_poor_qu.name = 'overall_poor_qu'\n",
    "overall_good_qu = alldata.OverallQual.copy()\n",
    "overall_good_qu = overall_good_qu - 5\n",
    "overall_good_qu[overall_good_qu<0] = 0\n",
    "overall_good_qu.name = 'overall_good_qu'\n",
    " \n",
    "# 处理OverallCond ：将该属性分成两个子属性，以5为分界线，大于5及小于5的再分别以序列\n",
    "overall_poor_cond = alldata.OverallCond.copy()# Series类型\n",
    "overall_poor_cond = 5 - overall_poor_cond\n",
    "overall_poor_cond[overall_poor_cond<0] = 0\n",
    "overall_poor_cond.name = 'overall_poor_cond'\n",
    "overall_good_cond = alldata.OverallCond.copy()\n",
    "overall_good_cond = overall_good_cond - 5\n",
    "overall_good_cond[overall_good_cond<0] = 0\n",
    "overall_good_cond.name = 'overall_good_cond'\n",
    " \n",
    "# 处理ExterQual：将该属性分成两个子属性，以3为分界线，大于3及小于3的再分别以序列\n",
    "exter_poor_qu = alldata.ExterQual.copy()\n",
    "exter_poor_qu[exter_poor_qu<3] = 1\n",
    "exter_poor_qu[exter_poor_qu>=3] = 0\n",
    "exter_poor_qu.name = 'exter_poor_qu'\n",
    "exter_good_qu = alldata.ExterQual.copy()\n",
    "exter_good_qu[exter_good_qu<=3] = 0\n",
    "exter_good_qu[exter_good_qu>3] = 1\n",
    "exter_good_qu.name = 'exter_good_qu'\n",
    " \n",
    "# 处理ExterCond：将该属性分成两个子属性，以3为分界线，大于3及小于3的再分别以序列\n",
    "exter_poor_cond = alldata.ExterCond.copy()\n",
    "exter_poor_cond[exter_poor_cond<3] = 1\n",
    "exter_poor_cond[exter_poor_cond>=3] = 0\n",
    "exter_poor_cond.name = 'exter_poor_cond'\n",
    "exter_good_cond = alldata.ExterCond.copy()\n",
    "exter_good_cond[exter_good_cond<=3] = 0\n",
    "exter_good_cond[exter_good_cond>3] = 1\n",
    "exter_good_cond.name = 'exter_good_cond'\n",
    " \n",
    "# 处理BsmtCond：将该属性分成两个子属性，以3为分界线，大于3及小于3的再分别以序列\n",
    "bsmt_poor_cond = alldata.BsmtCond.copy()\n",
    "bsmt_poor_cond[bsmt_poor_cond<3] = 1\n",
    "bsmt_poor_cond[bsmt_poor_cond>=3] = 0\n",
    "bsmt_poor_cond.name = 'bsmt_poor_cond'\n",
    "bsmt_good_cond = alldata.BsmtCond.copy()\n",
    "bsmt_good_cond[bsmt_good_cond<=3] = 0\n",
    "bsmt_good_cond[bsmt_good_cond>3] = 1\n",
    "bsmt_good_cond.name = 'bsmt_good_cond'\n",
    " \n",
    "# 处理GarageQual：将该属性分成两个子属性，以3为分界线，大于3及小于3的再分别以序列\n",
    "garage_poor_qu = alldata.GarageQual.copy()\n",
    "garage_poor_qu[garage_poor_qu<3] = 1\n",
    "garage_poor_qu[garage_poor_qu>=3] = 0\n",
    "garage_poor_qu.name = 'garage_poor_qu'\n",
    "garage_good_qu = alldata.GarageQual.copy()\n",
    "garage_good_qu[garage_good_qu<=3] = 0\n",
    "garage_good_qu[garage_good_qu>3] = 1\n",
    "garage_good_qu.name = 'garage_good_qu'\n",
    " \n",
    "# 处理GarageCond：将该属性分成两个子属性，以3为分界线，大于3及小于3的再分别以序列\n",
    "garage_poor_cond = alldata.GarageCond.copy()\n",
    "garage_poor_cond[garage_poor_cond<3] = 1\n",
    "garage_poor_cond[garage_poor_cond>=3] = 0\n",
    "garage_poor_cond.name = 'garage_poor_cond'\n",
    "garage_good_cond = alldata.GarageCond.copy()\n",
    "garage_good_cond[garage_good_cond<=3] = 0\n",
    "garage_good_cond[garage_good_cond>3] = 1\n",
    "garage_good_cond.name = 'garage_good_cond'\n",
    " \n",
    "# 处理KitchenQual：将该属性分成两个子属性，以3为分界线，大于3及小于3的再分别以序列\n",
    "kitchen_poor_qu = alldata.KitchenQual.copy()\n",
    "kitchen_poor_qu[kitchen_poor_qu<3] = 1\n",
    "kitchen_poor_qu[kitchen_poor_qu>=3] = 0\n",
    "kitchen_poor_qu.name = 'kitchen_poor_qu'\n",
    "kitchen_good_qu = alldata.KitchenQual.copy()\n",
    "kitchen_good_qu[kitchen_good_qu<=3] = 0\n",
    "kitchen_good_qu[kitchen_good_qu>3] = 1\n",
    "kitchen_good_qu.name = 'kitchen_good_qu'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "#将构造的属性合并\n",
    "qu_list = pd.concat((overall_poor_qu, overall_good_qu, overall_poor_cond, overall_good_cond, exter_poor_qu,\n",
    "                     exter_good_qu, exter_poor_cond, exter_good_cond, bsmt_poor_cond, bsmt_good_cond, garage_poor_qu,\n",
    "                     garage_good_qu, garage_poor_cond, garage_good_cond, kitchen_poor_qu, kitchen_good_qu), axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "#对与时间相关属性处理\n",
    "Xremoded = (alldata['YearBuilt']!=alldata['YearRemodAdd'])*1 #(@@@@@)\n",
    "Xrecentremoded = (alldata['YearRemodAdd']>=alldata['YrSold'])*1 #(@@@@@)\n",
    "XnewHouse = (alldata['YearBuilt']>=alldata['YrSold'])*1 #(@@@@@)\n",
    "XHouseAge = 2010 - alldata['YearBuilt']\n",
    "XTimeSinceSold = 2010 - alldata['YrSold']\n",
    "XYearSinceRemodel = alldata['YrSold'] - alldata['YearRemodAdd']\n",
    " \n",
    "Xremoded.name='Xremoded'\n",
    "Xrecentremoded.name='Xrecentremoded'\n",
    "XnewHouse.name='XnewHouse'\n",
    "XTimeSinceSold.name='XTimeSinceSold'\n",
    "XYearSinceRemodel.name='XYearSinceRemodel'\n",
    "XHouseAge.name='XHouseAge'\n",
    " \n",
    "year_list = pd.concat((Xremoded,Xrecentremoded,XnewHouse,XHouseAge,XTimeSinceSold,XYearSinceRemodel),axis=1)"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "构造新属性'price_category'\n",
    "\n",
    "此处利用SVM支持向量机构造新属性\n",
    "\n",
    "class sklearn.svm.SVC(C=1.0, kernel=’rbf’, degree=3, gamma=’auto’, coef0=0.0, \n",
    "shrinking=True, probability=False, tol=0.001, cache_size=200, class_weight=None,\n",
    "verbose=False, max_iter=-1, decision_function_shape=’ovr’, random_state=None)\n",
    "SVC参数解释 \n",
    "（1）C: 目标函数的惩罚系数C，用来平衡分类间隔margin和错分样本的，default C = 1.0； \n",
    "（2）kernel：参数选择有RBF, Linear, Poly, Sigmoid, 默认的是\"RBF\"; \n",
    "（3）degree：if you choose 'Poly' in param 2, this is effective, degree决定了多项式的最高次幂； \n",
    "（4）gamma：核函数的系数('Poly', 'RBF' and 'Sigmoid'), 默认是gamma = 1 / n_features; \n",
    "（5）coef0：核函数中的独立项，'RBF' and 'Poly'有效； \n",
    "（6）probablity: 可能性估计是否使用(true or false)； \n",
    "（7）shrinking：是否进行启发式； \n",
    "（8）tol（default = 1e - 3）: svm结束标准的精度; \n",
    "（9）cache_size: 制定训练所需要的内存（以MB为单位）； \n",
    "（10）class_weight: 每个类所占据的权重，不同的类设置不同的惩罚参数C, 缺省的话自适应； \n",
    "（11）verbose: 跟多线程有关，不大明白啥意思具体； \n",
    "（12）max_iter: 最大迭代次数，default = 1， if max_iter = -1, no limited; \n",
    "（13）decision_function_shape ： ‘ovo’ 一对一, ‘ovr’ 多对多  or None 无, default=None \n",
    "（14）random_state ：用于概率估计的数据重排时的伪随机数生成器的种子。 \n",
    " ps：7,8,9一般不考虑。 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.svm import SVC\n",
    "svm = SVC(C=100, gamma=0.0001, kernel='rbf')\n",
    " \n",
    "pc = pd.Series(np.zeros(train.shape[0]))\n",
    "pc[:] = 'pc1'\n",
    "pc[train.SalePrice >= 150000] = 'pc2'\n",
    "pc[train.SalePrice >= 220000] = 'pc3'\n",
    "columns_for_pc = ['Exterior1st', 'Exterior2nd', 'RoofMatl', 'Condition1', 'Condition2', 'BldgType']\n",
    "X_t = pd.get_dummies(train.loc[:, columns_for_pc], sparse=True)\n",
    "svm.fit(X_t, pc)# 训练\n",
    "p = train.SalePrice/100000\n",
    " \n",
    "price_category = pd.DataFrame(np.zeros((alldata.shape[0],1)), columns=['pc'])\n",
    "X_t = pd.get_dummies(alldata.loc[:, columns_for_pc], sparse=True)\n",
    "pc_pred = svm.predict(X_t) # 预测\n",
    " \n",
    "price_category[pc_pred=='pc2'] = 1\n",
    "price_category[pc_pred=='pc3'] = 2\n",
    "price_category.name='price_category'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- 连续数据离散化"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "year_map = pd.concat(pd.Series('YearGroup' + str(i+1), index=range(1871+i*20,1891+i*20)) for i in range(0, 7))\n",
    "# 将年份对应映射\n",
    "# alldata.GarageYrBlt = alldata.GarageYrBlt.map(year_map) # 在数据填充时已经完成该转换了（因为必须先转化后再填充，否则会出错（可以想想到底为什么呢？））\n",
    "alldata.YearBuilt = alldata.YearBuilt.map(year_map)\n",
    "alldata.YearRemodAdd = alldata.YearRemodAdd.map(year_map)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 4.2 处理数值型属性"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 简单函数 规范化 按照比例缩放\n",
    "numeric_feats = alldata.dtypes[alldata.dtypes != \"object\"].index\n",
    "t = alldata[numeric_feats].quantile(.75) # 取四分之三分位\n",
    "use_75_scater = t[t != 0].index\n",
    "alldata[use_75_scater] = alldata[use_75_scater]/alldata[use_75_scater].quantile(.75)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 标准化数据使符合正态分布\n",
    "from scipy.special import boxcox1p\n",
    " \n",
    "t = ['LotFrontage', 'LotArea', 'MasVnrArea', 'BsmtFinSF1', 'BsmtFinSF2', 'BsmtUnfSF', 'TotalBsmtSF',\n",
    "     '1stFlrSF', '2ndFlrSF', 'LowQualFinSF', 'GrLivArea', 'GarageArea', 'WoodDeckSF', 'OpenPorchSF',\n",
    "     'EnclosedPorch', '3SsnPorch', 'ScreenPorch', 'PoolArea', 'MiscVal']\n",
    "# alldata.loc[:, t] = np.log1p(alldata.loc[:, t])\n",
    "train[\"SalePrice\"] = np.log1p(train[\"SalePrice\"]) # 对于SalePrice 采用log1p较好---np.expm1(clf1.predict(X_test))\n",
    " \n",
    "lam = 0.15 # 100 * (1-lam)% confidence\n",
    "for feat in t:\n",
    "    alldata[feat] = boxcox1p(alldata[feat], lam)  # 对于其他属性，采用boxcox1p较好"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 将标称型变量二值化\n",
    "X = pd.get_dummies(alldata)\n",
    "X = X.fillna(X.mean())\n",
    " \n",
    "X = X.drop('Condition2_PosN', axis=1)\n",
    "X = X.drop('MSZoning_C (all)', axis=1)\n",
    "X = X.drop('MSSubClass_160', axis=1)\n",
    "X= pd.concat((X, newer_dwelling, season, year_list ,qu_list,MasVnrType_Any, \\\n",
    "              price_category,SaleCondition_PriceDown,Neighborhood_Good), axis=1)#SaleCondition_PriceDown,Neighborhood_Good已经加入"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 创建新属性\n",
    "from itertools import product, chain\n",
    "# chain(iter1, iter2, ..., iterN):\n",
    "# 给出一组迭代器(iter1, iter2, ..., iterN)，此函数创建一个新迭代器来将所有的迭代器链接起来，\n",
    "# 返回的迭代器从iter1开始生成项，知道iter1被用完，然后从iter2生成项，这一过程会持续到iterN中所有的项都被用完。\n",
    "def poly(X):\n",
    "    areas = ['LotArea', 'TotalBsmtSF', 'GrLivArea', 'GarageArea', 'BsmtUnfSF']  # 5个\n",
    "    t = chain(qu_list.axes[1].get_values(),year_list.axes[1].get_values(),ordinalList,\n",
    "              ['MasVnrType_Any'])  #,'Neighborhood_Good','SaleCondition_PriceDown'\n",
    "    for a, t in product(areas, t):\n",
    "        x = X.loc[:, [a, t]].prod(1) # 返回各维数组的乘积\n",
    "        x.name = a + '_' + t\n",
    "        yield x\n",
    "# 带有 yield 的函数在 Python 中被称之为 generator（生成器）\n",
    "# 一个带有 yield 的函数就是一个 generator，它和普通函数不同，生成一个 generator 看起来像函数调用，\n",
    "# 但不会执行任何函数代码，直到对其调用 next()（在 for 循环中会自动调用 next()）才开始执行。\n",
    "# 虽然执行流程仍按函数的流程执行，但每执行到一个 yield 语句就会中断，并返回一个迭代值，\n",
    "# 下次执行时从 yield 的下一个语句继续执行。看起来就好像一个函数在正常执行的过程中被 yield 中断了数次，\n",
    "# 每次中断都会通过 yield 返回当前的迭代值。\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "XP = pd.concat(poly(X), axis=1) # (2917, 155)\n",
    "X = pd.concat((X, XP), axis=1) # (2917, 466)\n",
    "X_train = X[:train.shape[0]]\n",
    "X_test = X[train.shape[0]:]\n",
    "print(X_train.shape)#(1458, 460)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'X_train' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-18-fb2c45aabf58>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m      1\u001b[0m \u001b[0mY\u001b[0m\u001b[1;33m=\u001b[0m \u001b[0mtrain\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mSalePrice\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 2\u001b[1;33m \u001b[0mtrain_now\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mpd\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mconcat\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mX_train\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mY\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0maxis\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      3\u001b[0m \u001b[0mtest_now\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mX_test\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      4\u001b[0m \u001b[1;31m# 将处理后的结果进行保存，用于接下来构建模型\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      5\u001b[0m \u001b[0mtrain_now\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mto_csv\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'./data_change/data_changetrain_afterchange.csv'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mNameError\u001b[0m: name 'X_train' is not defined"
     ]
    }
   ],
   "source": [
    "Y= train.SalePrice\n",
    "train_now = pd.concat([X_train,Y], axis=1)\n",
    "test_now = X_test\n",
    "# 将处理后的结果进行保存，用于接下来构建模型\n",
    "train_now.to_csv('./data_change/data_changetrain_afterchange.csv')\n",
    "test_now.to_csv('./data_change/test_afterchange.csv')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
